{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "resnet.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "sSw_-JAWi_nk"
      },
      "source": [
        "# -*- coding:utf-8 -*-\n",
        "# handwritten digits recognition\n",
        "# Data: MINIST\n",
        "# model: resnet\n",
        "# date: 2021.10.8 14:18\n",
        "\n",
        "import math\n",
        "import torch\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "import torch.nn as nn\n",
        "import torch.utils.data as Data\n",
        "import torch.optim as optim\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wAMLAt-7jlnL"
      },
      "source": [
        "train_curve = []\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MjTOGiQaqw4A"
      },
      "source": [
        "# param\n",
        "batch_size = 100\n",
        "n_class = 10\n",
        "padding_size = 15\n",
        "epoches = 10"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pz0nsjE-jwVt"
      },
      "source": [
        "train_dataset = torchvision.datasets.MNIST('./data/', train=True, transform=transforms.ToTensor(), download=True)\n",
        "test_dataset = torchvision.datasets.MNIST('./data/', train=False, transform=transforms.ToTensor(), download=False)\n",
        "train = Data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, num_workers=5)\n",
        "test = Data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False, num_workers=5)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5t9GsIw5CCBx"
      },
      "source": [
        "def gelu(x):\n",
        "  \"Implementation of the gelu activation function by Hugging Face\"\n",
        "  return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VHOBzUgztqQE"
      },
      "source": [
        "class ResBlock(nn.Module):\n",
        "  def __init__(self, in_size, out_size1, out_size2):\n",
        "    super(ResBlock, self).__init__()\n",
        "    self.conv1 = nn.Conv2d(\n",
        "        in_channels = in_size,\n",
        "        out_channels = out_size1,\n",
        "        kernel_size = 3,\n",
        "        stride = 2,\n",
        "        padding = padding_size\n",
        "    )\n",
        "    self.conv2 = nn.Conv2d(\n",
        "        in_channels = out_size1,\n",
        "        out_channels = out_size2,\n",
        "        kernel_size = 3,\n",
        "        stride = 2,\n",
        "        padding = padding_size\n",
        "    )\n",
        "    self.batchnorm1 = nn.BatchNorm2d(out_size1)\n",
        "    self.batchnorm2 = nn.BatchNorm2d(out_size2)\n",
        "  \n",
        "  def conv(self, x):\n",
        "    x = gelu(self.batchnorm1(self.conv1(x)))\n",
        "    x = gelu(self.batchnorm2(self.conv2(x)))\n",
        "    return x\n",
        "  \n",
        "  def forward(self, x):\n",
        "    return x + self.conv(x)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rWj7lSNRqj7a"
      },
      "source": [
        "# resnet\n",
        "class Resnet(nn.Module):\n",
        "  def __init__(self, n_class = n_class):\n",
        "    super(Resnet, self).__init__()\n",
        "    self.res1 = ResBlock(1, 8, 16)\n",
        "    self.res2 = ResBlock(16, 32, 16)\n",
        "    self.conv = nn.Conv2d(\n",
        "        in_channels = 16,\n",
        "        out_channels = n_class,\n",
        "        kernel_size = 3,\n",
        "        stride = 2,\n",
        "        padding = padding_size\n",
        "    )\n",
        "    self.batchnorm = nn.BatchNorm2d(n_class)\n",
        "    self.max_pooling = nn.AdaptiveAvgPool2d(1)\n",
        "\n",
        "  def forward(self, x):\n",
        "    x = x.view(-1, 1, 28, 28)\n",
        "    x = self.res1(x)\n",
        "    x = self.res2(x)\n",
        "    x = self.max_pooling(self.batchnorm(self.conv(x)))\n",
        "\n",
        "    return x.view(x.size(0), -1)\n"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jk63HsdJ5Pv_",
        "outputId": "d1a36e02-2bc5-46c1-ab8d-cf04ef5b3e3f"
      },
      "source": [
        "resnet = Resnet().to(device)\n",
        "resnet"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Resnet(\n",
              "  (res1): ResBlock(\n",
              "    (conv1): Conv2d(1, 8, kernel_size=(3, 3), stride=(2, 2), padding=(15, 15))\n",
              "    (conv2): Conv2d(8, 16, kernel_size=(3, 3), stride=(2, 2), padding=(15, 15))\n",
              "    (batchnorm1): BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (batchnorm2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "  )\n",
              "  (res2): ResBlock(\n",
              "    (conv1): Conv2d(16, 32, kernel_size=(3, 3), stride=(2, 2), padding=(15, 15))\n",
              "    (conv2): Conv2d(32, 16, kernel_size=(3, 3), stride=(2, 2), padding=(15, 15))\n",
              "    (batchnorm1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (batchnorm2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "  )\n",
              "  (conv): Conv2d(16, 10, kernel_size=(3, 3), stride=(2, 2), padding=(15, 15))\n",
              "  (batchnorm): BatchNorm2d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "  (max_pooling): AdaptiveAvgPool2d(output_size=1)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "br_L1zb65YZr"
      },
      "source": [
        "loss_fn = nn.CrossEntropyLoss()\n",
        "optimizer = optim.SGD(params=resnet.parameters(), lr=1e-2, momentum=0.9)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2_DGwLWC6x4p",
        "outputId": "d0a9cc2d-a12a-4e15-972b-2a7bd1bef09d"
      },
      "source": [
        "# train\n",
        "total_step = len(train)\n",
        "sum_loss = 0\n",
        "for epoch in range(epoches):\n",
        "  for i, (images, targets) in enumerate(train):\n",
        "    optimizer.zero_grad()\n",
        "    images = images.to(device)\n",
        "    targets = targets.to(device)\n",
        "    preds = resnet(images)\n",
        "    \n",
        "    loss = loss_fn(preds, targets)\n",
        "    sum_loss += loss.item()\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    if (i+1)%100==0:\n",
        "      print('[{}|{}] step:{}/{} loss:{:.4f}'.format(epoch+1, epoches, i+1, total_step, loss.item()))\n",
        "  train_curve.append(sum_loss)\n",
        "  sum_loss = 0\n",
        "    "
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 5 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1|10] step:100/600 loss:0.0104\n",
            "[1|10] step:200/600 loss:0.0106\n",
            "[1|10] step:300/600 loss:0.0732\n",
            "[1|10] step:400/600 loss:0.0317\n",
            "[1|10] step:500/600 loss:0.1365\n",
            "[1|10] step:600/600 loss:0.0139\n",
            "[2|10] step:100/600 loss:0.1351\n",
            "[2|10] step:200/600 loss:0.0275\n",
            "[2|10] step:300/600 loss:0.0894\n",
            "[2|10] step:400/600 loss:0.0616\n",
            "[2|10] step:500/600 loss:0.0297\n",
            "[2|10] step:600/600 loss:0.0832\n",
            "[3|10] step:100/600 loss:0.0885\n",
            "[3|10] step:200/600 loss:0.0124\n",
            "[3|10] step:300/600 loss:0.0781\n",
            "[3|10] step:400/600 loss:0.0477\n",
            "[3|10] step:500/600 loss:0.0048\n",
            "[3|10] step:600/600 loss:0.0412\n",
            "[4|10] step:100/600 loss:0.0146\n",
            "[4|10] step:200/600 loss:0.0193\n",
            "[4|10] step:300/600 loss:0.0526\n",
            "[4|10] step:400/600 loss:0.0025\n",
            "[4|10] step:500/600 loss:0.0876\n",
            "[4|10] step:600/600 loss:0.0551\n",
            "[5|10] step:100/600 loss:0.0240\n",
            "[5|10] step:200/600 loss:0.0036\n",
            "[5|10] step:300/600 loss:0.0077\n",
            "[5|10] step:400/600 loss:0.0169\n",
            "[5|10] step:500/600 loss:0.0079\n",
            "[5|10] step:600/600 loss:0.0342\n",
            "[6|10] step:100/600 loss:0.0029\n",
            "[6|10] step:200/600 loss:0.0772\n",
            "[6|10] step:300/600 loss:0.0368\n",
            "[6|10] step:400/600 loss:0.0408\n",
            "[6|10] step:500/600 loss:0.0082\n",
            "[6|10] step:600/600 loss:0.0354\n",
            "[7|10] step:100/600 loss:0.0042\n",
            "[7|10] step:200/600 loss:0.0313\n",
            "[7|10] step:300/600 loss:0.0376\n",
            "[7|10] step:400/600 loss:0.0500\n",
            "[7|10] step:500/600 loss:0.0020\n",
            "[7|10] step:600/600 loss:0.0330\n",
            "[8|10] step:100/600 loss:0.0175\n",
            "[8|10] step:200/600 loss:0.0473\n",
            "[8|10] step:300/600 loss:0.0029\n",
            "[8|10] step:400/600 loss:0.0147\n",
            "[8|10] step:500/600 loss:0.0300\n",
            "[8|10] step:600/600 loss:0.0006\n",
            "[9|10] step:100/600 loss:0.0401\n",
            "[9|10] step:200/600 loss:0.0286\n",
            "[9|10] step:300/600 loss:0.0258\n",
            "[9|10] step:400/600 loss:0.0167\n",
            "[9|10] step:500/600 loss:0.0201\n",
            "[9|10] step:600/600 loss:0.0574\n",
            "[10|10] step:100/600 loss:0.0066\n",
            "[10|10] step:200/600 loss:0.0097\n",
            "[10|10] step:300/600 loss:0.0616\n",
            "[10|10] step:400/600 loss:0.0515\n",
            "[10|10] step:500/600 loss:0.0019\n",
            "[10|10] step:600/600 loss:0.0381\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qbngK_gf9RBl",
        "outputId": "cdbcda1b-3562-48fd-b52e-a77e141e4d22"
      },
      "source": [
        "# test\n",
        "resnet.eval()\n",
        "with torch.no_grad():\n",
        "  correct = 0\n",
        "  total = 0\n",
        "  for images, labels in test:\n",
        "    images = images.to(device)\n",
        "    labels = labels.to(device)\n",
        "    outputs = resnet(images)\n",
        "    _, maxIndexes = torch.max(outputs, dim=1)\n",
        "    correct += (maxIndexes==labels).sum().item()\n",
        "    total += labels.size(0)\n",
        "  \n",
        "  print('in 1w test_data correct rate = {:.4f}'.format((correct/total)*100))"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 5 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "in 1w test_data correct rate = 98.5100\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "418J4zPhBlu6",
        "outputId": "041cfecc-0197-40fa-c716-3a1505515baf"
      },
      "source": [
        "pd.DataFrame(train_curve).plot() # loss曲线"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f046fd61950>"
            ]
          },
          "metadata": {},
          "execution_count": 24
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3xcZb3v8c9vZnJp0qSXSXpNSVrbDVIQKYEW2ZvjEeS2laKClmu5bKoCHtjsfRRf7O11q7jdiiIIh1KkYCkgisUtt1pQz1FaSKGUXoCG0tKUXtL7NU0y85w/1pNmmiZtkklmklnf9+uV16z1rDWzfp1Mv2vlWc+sZc45REQkHCLZLkBERDJHoS8iEiIKfRGREFHoi4iEiEJfRCREYtku4EjKyspcVVVVtssQEelXFi9evMU5V97esj4d+lVVVdTU1GS7DBGRfsXM1na0TN07IiIhotAXEQkRhb6ISIj06T59EZFsaWpqoq6ujoaGhmyX0qHCwkIqKirIy8vr9HMU+iIi7airq6OkpISqqirMLNvlHMY5x9atW6mrq2Ps2LGdfp66d0RE2tHQ0EA8Hu+TgQ9gZsTj8S7/JaLQFxHpQF8N/Bbdqe+ooW9mD5rZZjNbltL2IzN7y8yWmtlTZjY4ZdnXzazWzN42s3NT2s/zbbVmdluXK+2CA80JfvDsSuq27+vNzYiI9DudOdJ/CDivTdt84ATn3EeAd4CvA5jZ8cA0YKJ/zi/MLGpmUeAe4HzgeOBSv26v2LzrAI8ufJ+bHn2dxuZkb21GRKTXPffccxx77LGMHz+eO+64I+3XO2roO+f+Amxr0/aCc67Zzy4EKvz0VOAx59wB59x7QC1wmv+pdc6tds41Ao/5dXvFmKFF/OiSj7Bk3Q6+/8zK3tqMiEivSiQS3HjjjTz77LOsWLGCuXPnsmLFirResyf69K8FnvXTo4F1KcvqfFtH7YcxsxlmVmNmNfX19d0u6rwTRnLtGWN56G9r+MPSDd1+HRGRbHnllVcYP34848aNIz8/n2nTpjFv3ry0XjOtIZtmdjvQDMxJq4oUzrn7gfsBqqur07qX423nH8fr67bztd8s5cMjSxhXPrBHahSRcPn275ez4oNdPfqax48q5ZufnnjEddavX8+YMWMOzldUVLBo0aK0ttvtI30zuxr4FHC5a73R7npgTMpqFb6to/ZelR+LcM9lk8iLGjfMeY2GpkRvb1JEpE/r1pG+mZ0HfBX4H8651CEyTwOPmtlPgFHABOAVwIAJZjaWIOynAZelU3hnjRo8gDu/8FGueehVvjFvGf958UmZ2KyI5JCjHZH3ltGjR7NuXWvPeF1dHaNHt9sz3mmdGbI5F3gZONbM6szsOuBuoASYb2ZLzOw+AOfccuAJYAXwHHCjcy7hT/reBDwPrASe8OtmxMePHcZN/3M8T9TU8euadUd/gohIH3DqqaeyatUq3nvvPRobG3nssce48MIL03rNox7pO+cubad51hHW/x7wvXbanwGe6VJ1PeiWs/+OxWu38+/zlnFixSCOG1GarVJERDolFotx9913c+6555JIJLj22muZODG9vzpC843caMT42bSTKS3M44ZfvcbuhqZslyQiclQXXHAB77zzDu+++y6333572q8XmtAHKC8p4OeXnszabfu47bdv0nr+WUQkHEIV+gCTx8X513OO5Q9LN/Dwyx3eUUxEJCeFLvQBvnjmOM46bhj/8YcVLFm3I9vliEgf1dd7A7pTXyhDPxIxfvz5kxhWUsiNc15jx77GbJckIn1MYWEhW7du7bPB33I9/cLCwi49L7Q3URlclM8vLp/EJfe9zK1PvMEDV1UTifTty6iKSOZUVFRQV1dHOpeD6W0td87qitCGPsBJYwbzb5/6MN+Yt5z7/vIuN3x8fLZLEpE+Ii8vr0t3pOovQtm9k+rKKZV8+qRR/Nfzb7Nw9dZslyMi0qtCH/pmxg8+eyJVZcV8Ze7rbN7dd2+CLCKSrtCHPsDAghj3Xn4KuxuauHnuEhLJvnniRkQkXQp979gRJfzHRSfy8uqt3Dn/nWyXIyLSKxT6KS4+pYIvVI/h7pdqefGtTdkuR0Skxyn02/j21IlMHFXKDXNe4+V3dWJXRHKLQr+Nwrwos689jTFDirhu9qu8umbb0Z8kItJPKPTbUTawgDnXT2bEoEKufvAVFq/dnu2SRER6hEK/A8NKCpl7/RSGlQbB/4au0SMiOUChfwTDSwt59PrJDCnO58pZi1i2fme2SxIRSYtC/yhGDhrAo9dPpqQwjytmLWLFB7uyXZKISLcp9DuhYkgRc6+fwoC8KFfMWsTbG3dnuyQRkW5R6HfSMfEg+POixuUPLKR2s4JfRPofhX4XVJUV8+j1UwDj0pmLWF2/J9sliYh0iUK/iz5UPpC5108mmXRcNnMRa7fuzXZJIiKdptDvhgnDS5hz/WQONCe4bOYi1m3bl+2SREQ6RaHfTceNKOVX/zSZPQeauXTmQtbv2J/tkkREjuqooW9mD5rZZjNbltI21Mzmm9kq/zjEt5uZ3WVmtWa21MwmpTxnul9/lZlN751/TmZNHDWIX103mZ37m7hs5kI27tS1+EWkb+vMkf5DwHlt2m4DFjjnJgAL/DzA+cAE/zMDuBeCnQTwTWAycBrwzZYdRX93YsUgHr72NLbuaeTSmQvZvEvBLyJ911FD3zn3F6DtVcemArP99GzgopT2h11gITDYzEYC5wLznXPbnHPbgfkcviPpt04+ZggPXXMqm3Y1cOnMhdTvPpDtkkRE2tXdPv3hzrkNfnojMNxPjwbWpaxX59s6aj+Mmc0wsxozq+nLd6Fvq7pqKL+8+lQ+2BEEv7p6RKQvSvtErnPOAT12f0Hn3P3OuWrnXHV5eXlPvWxGTB4X55fXnMrGnQ187t6/aRy/iPQ53Q39Tb7bBv+42bevB8akrFfh2zpqzzlTxsV5bMYUGpoSXHLfy7xZp4u0iUjf0d3QfxpoGYEzHZiX0n6VH8UzBdjpu4GeB84xsyH+BO45vi0nnTB6EL/+0ukU5kW5dOZC/vbulmyXJCICdG7I5lzgZeBYM6szs+uAO4BPmtkq4Gw/D/AMsBqoBWYCNwA457YB3wVe9T/f8W05a1z5QH7z5Y8xanAhVz/4Ks8t25jtkkREsKBLvm+qrq52NTU12S4jLTv2NXLNQ6/yxrod/OCzJ/KFU4/JdkkikuPMbLFzrrq9ZfpGbi8bXJTPnH+azD9MKOdrv3mTe//0Ln15RysiuU2hnwFF+TFmXlXNhSeN4ofPvcX3n1mp4BeRrIhlu4CwyI9F+OkXPsqQojxm/t/32L6viTs+eyKxqPa7IpI5Cv0MikSMb104kaHFBdz5x3fYsa+Juy87mcK8aLZLE5GQ0GFmhpkZN589ge9OnciCtzZx1YOvsKuhKdtliUhIKPSz5MrTq7hr2sm8/v52pv0fXa9HRDJDoZ9Fnz5pFLOmn8p7W/Zy8X1/081YRKTXKfSz7My/K2fO9cE1+T937994a+OubJckIjlMod8HTDpmCL/+4ulEzLjkvpf5a60u2yAivUOh30dMGF7Ck18+nVGDBnDVg68wZ9HabJckIjlIod+HVAwp4skvn86ZE8q4/allfOf3K0gk9SUuEek5Cv0+pqQwjwemn8q1Z4zlwb++x/UP17BbQzpFpIco9PugaMT4xqeP53ufOYE/v1PPxfe+rJE9ItIjFPp92OWTK5l9zWls2Lmfz/ziryxeuz3bJYlIP6fQ7+P+fkIZT914BgMLYlw6cyG/ez0nbzgmIhmi0O8HPlQ+kKduOIOTxwzmlseX8JMX3iapE7wi0g0K/X5iSHE+j1w3mS9Uj+GuF2v5ytzX2d+YyHZZItLP6Cqb/Uh+LMIdnzuR8cMG8v1nV1K3fR8zr6pmWGlhtksTkX5CR/r9jJlx/ZnjuP/KalZt3sPUe/7K8g92ZrssEeknFPr91CePH86TX/oYBlxy38u8sFw3XheRo1Po92PHjyrldzedwYThJXzxV4u578+6/66IHJlCv58bVlLI4zOmcMGJI7nj2bdYuHpbtksSkT5MoZ8DCvOifPvCiQC6NLOIHJFCP0fEi/MZWBBj7VZdrkFEOpZW6JvZP5vZcjNbZmZzzazQzMaa2SIzqzWzx80s369b4Odr/fKqnvgHSMDMqIwXsWbr3myXIiJ9WLdD38xGA/8LqHbOnQBEgWnAD4E7nXPjge3Adf4p1wHbffudfj3pQVXxYh3pi8gRpdu9EwMGmFkMKAI2AJ8AnvTLZwMX+empfh6//CwzszS3Lykq40Ws27aP5kQy26WISB/V7dB3zq0H/gt4nyDsdwKLgR3OuWa/Wh0w2k+PBtb55zb79eNtX9fMZphZjZnV1NfXd7e8UKoqK6Y56fhgR0O2SxGRPiqd7p0hBEfvY4FRQDFwXroFOefud85VO+eqy8vL0325UKmKFwOoX19EOpRO987ZwHvOuXrnXBPwW+AMYLDv7gGoAFquBbweGAPglw8CtqaxfWmjKl4EwFqFvoh0IJ3Qfx+YYmZFvm/+LGAF8BJwsV9nOjDPTz/t5/HLX3T6+miPKi8pYEBelDU6mSsiHUinT38RwQnZ14A3/WvdD3wNuNXMagn67Gf5p8wC4r79VuC2NOqWdrQM29SRvoh0JK1LKzvnvgl8s03zauC0dtZtAC5JZ3tydFXxYmrr92S7DBHpo/SN3BxTWVbE+1v3kdCdtUSkHQr9HFMVL6YxkWTjLg3bFJHDKfRzTGXLCJ4t6tcXkcMp9HNM61h9jeARkcMp9HPMiNJC8mMRjeARkXYp9HNMJGJUDtXVNkWkfQr9HFSpq22KSAcU+jmoyl9XX194FpG2FPo5qLKsmIamJJt3H8h2KSLSxyj0c1DLhdfWaNimiLSh0M9BLcM21a8vIm0p9HPQyEGF5EVNI3hE5DAK/RwUi0YYM6RIR/oichiFfo6qjGusvogcTqGfo1rG6mvYpoikUujnqKp4EXsONLN1b2O2SxGRPkShn6Mqy1pG8KiLR0RaKfRz1MGrbW7RyVwRaaXQz1GjBw8gGjEd6YvIIRT6OSo/FmH04AG6rr6IHEKhn8Mq40U60heRQyj0c1hVvFhH+iJyCIV+DquMF7FzfxM79mnYpogEFPo5TPfLFZG20gp9MxtsZk+a2VtmttLMTjezoWY238xW+cchfl0zs7vMrNbMlprZpJ75J0hHqsqCSyyrX19EWqR7pP8z4Dnn3HHAScBK4DZggXNuArDAzwOcD0zwPzOAe9PcthxFxZAizDRWX0RadTv0zWwQcCYwC8A51+ic2wFMBWb71WYDF/npqcDDLrAQGGxmI7tduRxVYV6UUYMG6EhfRA5K50h/LFAP/NLMXjezB8ysGBjunNvg19kIDPfTo4F1Kc+v822HMLMZZlZjZjX19fVplCegq22KyKHSCf0YMAm41zl3MrCX1q4cAFxwiccuXebROXe/c67aOVddXl6eRnkCwdU2dSJXRFqkE/p1QJ1zbpGff5JgJ7CppdvGP272y9cDY1KeX+HbpBdVxYvYtreRnfubsl2KiPQB3Q5959xGYJ2ZHeubzgJWAE8D033bdGCen34auMqP4pkC7EzpBpJeUumHbb6vo30RIeiiScdXgDlmlg+sBq4h2JE8YWbXAWuBz/t1nwEuAGqBfX5d6WUtwzbXbN3LiRWDslyNiGRbWqHvnFsCVLez6Kx21nXAjelsT7rumKEaqy8irfSN3BxXlB9jeGmBTuaKCKDQD4Xgfrk60hcRhX4oVMWLdKQvIoBCPxQq48XU7z7A3gPN2S5FRLJMoR8CLVfbXKujfZHQU+iHQGVcI3hEJKDQD4GW0Fe/vogo9EOgpDCPsoH5OtIXEYV+WAQXXlPoi4SdQj8kKuNFOpErIgr9sKiKF7NhZwMNTYlslyIiWaTQD4mWk7nvb9PRvkiYKfRDomWs/pot6tcXCTOFfkjoC1oiAgr90BhUlMfgojyN4BEJOYV+iARX29SRvkiYKfRDJLjapo70RcJMoR8ilfFiPtixnwPNGrYpElYK/RCpiheRdFC3fX+2SxGRLFHoh0jlwRE86uIRCSuFfohUtVxtc4tO5oqElUI/RIYW51NSENORvkiIKfRDxMyoLNP9ckXCTKEfMsFYfR3pi4RV2qFvZlEze93M/tvPjzWzRWZWa2aPm1m+by/w87V+eVW625auq4oXUbd9P02JZLZLEZEs6Ikj/ZuBlSnzPwTudM6NB7YD1/n264Dtvv1Ov55kWGW8mOak44MdGrYpEkZphb6ZVQD/CDzg5w34BPCkX2U2cJGfnurn8cvP8utLBh282qb69UVCKd0j/Z8CXwVa+griwA7nXLOfrwNG++nRwDoAv3ynX/8QZjbDzGrMrKa+vj7N8qStlmGb6tcXCaduh76ZfQrY7Jxb3IP14Jy73zlX7ZyrLi8v78mXFqC8pIABeVGN1RcJqVgazz0DuNDMLgAKgVLgZ8BgM4v5o/kKYL1ffz0wBqgzsxgwCNiaxvalG8zM3y9XR/oiYdTtI33n3NedcxXOuSpgGvCic+5y4CXgYr/adGCen37az+OXv+icc93dvnRfVbxYV9sUCaneGKf/NeBWM6sl6LOf5dtnAXHffitwWy9sWzqhsqyIddv2k0hqnysSNul07xzknPsT8Cc/vRo4rZ11GoBLemJ7kp6qeDGNiSQbdu6nYkhRtssRkQzSN3JDqPLgCB6dzBUJG4V+CLWO1Ve/vkjYKPRDaERpIfmxiI70RUJIoR9CkYhRObSINVt0pC8SNgr9kAqutqkjfZGwUeiHVFW8iLXb9pLUsE2RUFHoh1RlWTENTUk27z6Q7VJEJIMU+iF18H65GsEjEioK/ZBqGbapa/CIhItCP6RGDiokL2q6rr5IyCj0QyoWjTBmiK62KRI2Cv0Qq4wX6br6IiGj0A+xYKz+XnSFa5HwUOiHWFW8iL2NCbbsacx2KSKSIQr9EKss0wgekbBR6IdY69U21a8vEhYK/RAbPXgA0YjpSF8kRBT6IZYfizB68AAd6YuEiEI/5CrjGqsvEiYK/ZCrihfz3hYN2xQJC4V+yFXGi9jd0MyOfU3ZLkVEMkChH3K6X65IuCj0Q66qLLjEsu6iJRIOCv2QqxhShJmO9EXCotuhb2ZjzOwlM1thZsvN7GbfPtTM5pvZKv84xLebmd1lZrVmttTMJvXUP0K6rzAvyqhBA3SkLxIS6RzpNwP/4pw7HpgC3GhmxwO3AQuccxOABX4e4Hxggv+ZAdybxralB1XGi3SkLxIS3Q5959wG59xrfno3sBIYDUwFZvvVZgMX+empwMMusBAYbGYju1259Jjgaps60hcJgx7p0zezKuBkYBEw3Dm3wS/aCAz306OBdSlPq/NtbV9rhpnVmFlNfX19T5QnR1EVL2Lb3kZ27tewTZFcl3bom9lA4DfALc65XanLXPCNny5968c5d79zrto5V11eXp5uedIJlX7Y5vs62hfJeWmFvpnlEQT+HOfcb33zppZuG/+42bevB8akPL3Ct0mWtQzbVL++SO5LZ/SOAbOAlc65n6QsehqY7qenA/NS2q/yo3imADtTuoEkiyqHFmMGC1dvzXYpItLL0jnSPwO4EviEmS3xPxcAdwCfNLNVwNl+HuAZYDVQC8wEbkhj29KDBuRHuXzyMcxZ9D5/WKr9sEgui3X3ic65/wdYB4vPamd9B9zY3e1J7/r3Tx3Pig928b+ffIPxwwZy7IiSbJckIr1A38gVAApiUe694hSKC2LMeKSGnboAm0hOUujLQcNLC7nvikl8sGM/Nz/+OomkLrcskmsU+nKIUyqH8q0LJ/Knt+u5c/472S5HRHqYQl8Oc9lpxzDt1DHc/VItzy3TiV2RXKLQl8OYGd+eOpGPjhnMvzzxBqs27c52SSLSQxT60q6CWJT7rjiFAfkxZjyymF0NOrErkgsU+tKhEYMK+cXlk1i3bR///NgSkjqxK9LvKfTliE4bO5Rvfvp4Fry1mZ8uWJXtckQkTQp9OaorplRyySkV3LVgFS8s35jtckQkDQp9OSoz47sXncBHKgZx6xNvULt5T7ZLEpFuUuhLpxTmBSd2C2IRZjxSw26d2BXplxT60mmjBg/gnssnsXbrPm594g2d2BXphxT60iVTxsX5t3/8MPNXbOLul2qzXY6IdJFCX7rs6o9V8dlJo7nzj++wYOWmbJcjIl2g0JcuMzO+/5kTmTiqlFseW8Lqep3YFekvFPrSLS0ndvNiEWY8spg9B5qzXZKIdEK3b6IiUjGkiLsvO5krZ73C2T/+M8eNLGFc2UDGlRczrqyYceUDGV5aQHBnTRHpCxT6kpaPfaiMX1w+id+/8QGr6/eyaPU29jclDi4vzo8ytry4dWdQPtDvEIopytfHTyTT9L9O0nbuxBGcO3EEAMmkY+OuBt7bspfV9Xt4t34vq7fs5bX3t/P7pR/gUkZ5jhxUyNiyYirjxQwvLWB4aSHDSvxjaQHx4gKiEf2VINKTFPrSoyIRY9TgAYwaPIAzxpcdsqyhKcGarXtZXR/sEFbX7+XdLXt5fvlGtu1tPPy1DMpLChhWUsjw0gLK/eOwlMdhpQUMGpBHQSyibiSRTlDoS8YU5kU5bkQpx40oPWxZY3OSLXsOsGlXA5t3H2Czf9y0q4FNuw6wfkcDr7+/g63t7BwA8qJGaWEepQPyKCmM+engsXU+aCspaF2vOD9GYX6EAXlRBuRFiUU1tkFym0Jf+oT8WOTgXwhH0rJzaNkh1O8+wK6GJnY3NLNrfxO7GprZ3dDErv1NbNzV4KebDznPcCR5UQt2APnBTqAwZXpAXpTC/ChFvq0gFqEg5h/zDp/Oj7bfXhCLkB+LkBeNkB8NptWNJZmi0Jd+pbM7h7aaEsmUHUOwI9jd0MS+xgT7mxI0NCXY35hgn39saAra9ze2Pu7c3xSs69c70JSkoTlxyHmK7ooYh+wE8qIR8mJ2eFvUyG/ZofidR8vy/FhKm28vONgePdgWjRhmEDHzP8F3L6KR1ulI6vJI63Qsaoe8bp7fdixi6l7rJxT6Egp50QhDi/MZWpzfo6/rnKM56TjQnKSxOcmB5mBncKBlujnp5xN+eWt7Y3OSpoSjKZGkKRHMN/rppmZHY8LPN/vlvn3n/qZg3eZEsM7BbQePzVm4JpIZwc4nZQfUspM5dMcQ7FTMPydirTug4HX8jofWZWBEIxzc2eXH7JDXbXlM3XZrm/m/pILtRyOW8hjsAKNt26OHLk9nXxax4HUPPvaBv+gyHvpmdh7wMyAKPOCcuyPTNYj0FDMjLxockVOQ7WoCyWSww0jdEbXsUBqbkyQdJJ3DORdMJx0J53C+PXV5InnodHOy9XUbmw/dWR3ymLIDa9khJZLBNhzBNlwSEgT1tNTiAA5OO5LJoC2RDHaQbV83Wzu57jLjkJ1AsJOJtJkPHieOGsTPLz25x2vIaOibWRS4B/gkUAe8amZPO+dWZLIOkVwWiRiFkeB8RBi07ORadgStO4cEjf4vpkTSkUg6mpNJ/+hIJPyjb086R3PCtS5PBn+FpcM5/GslSSSDnVfrNl1KXYevc8zQrnVhdlamj/RPA2qdc6sBzOwxYCqg0BeRbgnbTi5dmR6fNhpYlzJf59sOMrMZZlZjZjX19fUZLU5EJNf1uUHJzrn7nXPVzrnq8vLybJcjIpJTMh3664ExKfMVvk1ERDIg06H/KjDBzMaaWT4wDXg6wzWIiIRWRk/kOueazewm4HmCIZsPOueWZ7IGEZEwy/g4fefcM8Azmd6uiIj0wRO5IiLSexT6IiIhYq4nrhbVS8ysHlibxkuUAVt6qJyepLq6RnV1jerqmlysq9I51+6Y9z4d+ukysxrnXHW262hLdXWN6uoa1dU1YatL3TsiIiGi0BcRCZFcD/37s11AB1RX16iurlFdXROqunK6T19ERA6V60f6IiKSQqEvIhIi/T70zew8M3vbzGrN7LZ2lheY2eN++SIzq8pATWPM7CUzW2Fmy83s5nbW+biZ7TSzJf7nG71dV8q215jZm367Ne0sNzO7y79nS81sUgZqOjblvVhiZrvM7JY262TkPTOzB81ss5ktS2kbambzzWyVfxzSwXOn+3VWmdn0DNT1IzN7y/+enjKzwR0894i/816o61tmtj7ld3VBB8894v/fXqjr8ZSa1pjZkg6e25vvV7v5kLHPmPP3v+yPPwQXbXsXGAfkA28Ax7dZ5wbgPj89DXg8A3WNBCb56RLgnXbq+jjw31l639YAZUdYfgHwLMH9qacAi7Lwe91I8AWTjL9nwJnAJGBZStt/Arf56duAH7bzvKHAav84xE8P6eW6zgFifvqH7dXVmd95L9T1LeBfO/F7PuL/356uq83yHwPfyML71W4+ZOoz1t+P9A/eftE51wi03H4x1VRgtp9+EjjLLJ372x+dc26Dc+41P70bWEmbO4T1cVOBh11gITDYzEZmcPtnAe8659L5Nna3Oef+Amxr05z6OZoNXNTOU88F5jvntjnntgPzgfN6sy7n3AvOuWY/u5DgHhUZ1cH71Rmd+f/bK3X5DPg8MLenttdZR8iHjHzG+nvoH/X2i6nr+P8cO4F4RqoDfHfSycCidhafbmZvmNmzZjYxUzUBDnjBzBab2Yx2lnfmfe1N0+j4P2O23rPhzrkNfnojMLyddbL9vl1L8Bdae472O+8NN/lupwc76KrI5vv1D8Am59yqDpZn5P1qkw8Z+Yz199Dv08xsIPAb4Bbn3K42i18j6L44Cfg58LsMlvb3zrlJwPnAjWZ2Zga3fUQW3FznQuDX7SzO5nt2kAv+zu5TY53N7HagGZjTwSqZ/p3fC3wI+CiwgaArpS+5lCMf5ff6+3WkfOjNz1h/D/3O3H7x4DpmFgMGAVt7uzAzyyP4hc5xzv227XLn3C7n3B4//QyQZ2ZlvV2X3956/7gZeIrgz+xU2byt5fnAa865TW0XZPM9Aza1dHH5x83trJOV983MrgY+BVzuw+Iwnfid9yjn3CbnXMI5lwRmdrC9bL1fMeCzwOMdrdPb71cH+ZCRz1h/D/3O3H7xaaDlDPfFwJXrcbMAAAFxSURBVIsd/cfoKb6/cBaw0jn3kw7WGdFybsHMTiP4XWRiZ1RsZiUt0wQnApe1We1p4CoLTAF2pvzZ2ds6PALL1nvmpX6OpgPz2lnneeAcMxviuzPO8W29xszOA74KXOic29fBOp35nfd0XanngD7TwfaydfvUs4G3nHN17S3s7ffrCPmQmc9Yb5ydzuQPwUiTdwhGAdzu275D8J8AoJCgq6AWeAUYl4Ga/p7gT7OlwBL/cwHwJeBLfp2bgOUEIxYWAh/L0Ps1zm/zDb/9lvcstTYD7vHv6ZtAdYZqKyYI8UEpbRl/zwh2OhuAJoI+0+sIzgMtAFYBfwSG+nWrgQdSnnut/6zVAtdkoK5agj7els9Zy0i1UcAzR/qd93Jdj/jPzlKCMBvZti4/f9j/396sy7c/1PKZSlk3k+9XR/mQkc+YLsMgIhIi/b17R0REukChLyISIgp9EZEQUeiLiISIQl9EJEQU+iIiIaLQFxEJkf8PYkCLlTQG8KMAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}